{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文本生成实战"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0\n",
      "sys.version_info(major=3, minor=6, micro=10, releaselevel='final', serial=0)\n",
      "matplotlib 3.1.2\n",
      "numpy 1.18.1\n",
      "pandas 0.25.3\n",
      "sklearn 0.22.1\n",
      "tensorflow 2.0.0\n",
      "tensorflow_core.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl,np,pd,sklearn,tf,keras:\n",
    "    print(module.__name__,module.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Physical GPUs, 1 Logical GPUs\n"
     ]
    }
   ],
   "source": [
    "def solve_cudnn_error():\n",
    "    gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "    if gpus:\n",
    "        try:\n",
    "            # Currently, memory growth needs to be the same across GPUs\n",
    "            for gpu in gpus:\n",
    "                tf.config.experimental.set_memory_growth(gpu, True)\n",
    "            logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "            print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "        except RuntimeError as e:\n",
    "            # Memory growth must be set before GPUs have been initialized\n",
    "            print(e)\n",
    "\n",
    "solve_cudnn_error()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据读取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1115394\n",
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You\n"
     ]
    }
   ],
   "source": [
    "input_filepath = './data/shakespeare.txt'\n",
    "text = open(input_filepath, 'r').read()\n",
    "\n",
    "print(len(text))\n",
    "print(text[0:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据处理\n",
    "1. 生成词表 generator vocab\n",
    "2. 建立字符到id的映射 build mapping char->id\n",
    "3. 将数据转化为id data->id_data\n",
    "4. 预测下一个字符的模型，abcd -> bcd\\<eos>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n",
      "['\\n', ' ', '!', '$', '&', \"'\", ',', '-', '.', '3', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n"
     ]
    }
   ],
   "source": [
    "# 1.generator vocab\n",
    "vocab = sorted(set(text)) # 把字符串存为set，则只剩下唯一的字符\n",
    "print(len(vocab))\n",
    "print(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\n': 0, ' ': 1, '!': 2, '$': 3, '&': 4, \"'\": 5, ',': 6, '-': 7, '.': 8, '3': 9, ':': 10, ';': 11, '?': 12, 'A': 13, 'B': 14, 'C': 15, 'D': 16, 'E': 17, 'F': 18, 'G': 19, 'H': 20, 'I': 21, 'J': 22, 'K': 23, 'L': 24, 'M': 25, 'N': 26, 'O': 27, 'P': 28, 'Q': 29, 'R': 30, 'S': 31, 'T': 32, 'U': 33, 'V': 34, 'W': 35, 'X': 36, 'Y': 37, 'Z': 38, 'a': 39, 'b': 40, 'c': 41, 'd': 42, 'e': 43, 'f': 44, 'g': 45, 'h': 46, 'i': 47, 'j': 48, 'k': 49, 'l': 50, 'm': 51, 'n': 52, 'o': 53, 'p': 54, 'q': 55, 'r': 56, 's': 57, 't': 58, 'u': 59, 'v': 60, 'w': 61, 'x': 62, 'y': 63, 'z': 64}\n"
     ]
    }
   ],
   "source": [
    "# 2.build mapping char->id\n",
    "char2idx = {char : idx for idx, char in enumerate(vocab)}\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n' ' ' '!' '$' '&' \"'\" ',' '-' '.' '3' ':' ';' '?' 'A' 'B' 'C' 'D' 'E'\n",
      " 'F' 'G' 'H' 'I' 'J' 'K' 'L' 'M' 'N' 'O' 'P' 'Q' 'R' 'S' 'T' 'U' 'V' 'W'\n",
      " 'X' 'Y' 'Z' 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o'\n",
      " 'p' 'q' 'r' 's' 't' 'u' 'v' 'w' 'x' 'y' 'z']\n"
     ]
    }
   ],
   "source": [
    "idx2char = np.array(vocab)\n",
    "print(idx2char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[18 47 56 57 58  1 15 47 58 47]\n",
      "First Citi\n"
     ]
    }
   ],
   "source": [
    "# 3.data->id_data\n",
    "text_as_int = np.array([char2idx[c] for c in text])\n",
    "print(text_as_int[0:10])\n",
    "print(text[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(18, shape=(), dtype=int32) F\n",
      "tf.Tensor(47, shape=(), dtype=int32) i\n",
      "tf.Tensor(\n",
      "[18 47 56 57 58  1 15 47 58 47 64 43 52 10  0 14 43 44 53 56 43  1 61 43\n",
      "  1 54 56 53 41 43 43 42  1 39 52 63  1 44 59 56 58 46 43 56  6  1 46 43\n",
      " 39 56  1 51 43  1 57 54 43 39 49  8  0  0 13 50 50 10  0 31 54 43 39 49\n",
      "  6  1 57 54 43 39 49  8  0  0 18 47 56 57 58  1 15 47 58 47 64 43 52 10\n",
      "  0 37 53 59  1], shape=(101,), dtype=int32)\n",
      "'First Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYou '\n",
      "tf.Tensor(\n",
      "[39 56 43  1 39 50 50  1 56 43 57 53 50 60 43 42  1 56 39 58 46 43 56  1\n",
      " 58 53  1 42 47 43  1 58 46 39 52  1 58 53  1 44 39 51 47 57 46 12  0  0\n",
      " 13 50 50 10  0 30 43 57 53 50 60 43 42  8  1 56 43 57 53 50 60 43 42  8\n",
      "  0  0 18 47 56 57 58  1 15 47 58 47 64 43 52 10  0 18 47 56 57 58  6  1\n",
      " 63 53 59  1 49], shape=(101,), dtype=int32)\n",
      "'are all resolved rather to die than to famish?\\n\\nAll:\\nResolved. resolved.\\n\\nFirst Citizen:\\nFirst, you k'\n"
     ]
    }
   ],
   "source": [
    "# 4.定义输入输出\n",
    "def split_input_target(id_text):\n",
    "    \"\"\"abcde -> abcd, bcde\"\"\"\n",
    "    return id_text[0:-1], id_text[1:]\n",
    "\n",
    "# 定义dataset\n",
    "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n",
    "seq_length = 100\n",
    "seq_dataset = char_dataset.batch(seq_length + 1, drop_remainder = True)\n",
    "\n",
    "# test\n",
    "for ch_id in char_dataset.take(2):\n",
    "    print(ch_id, idx2char[ch_id.numpy()])\n",
    "    \n",
    "for seq_id in seq_dataset.take(2):\n",
    "    print(seq_id)\n",
    "    print(repr(''.join(idx2char[seq_id.numpy()])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[18 47 56 57 58  1 15 47 58 47 64 43 52 10  0 14 43 44 53 56 43  1 61 43\n",
      "  1 54 56 53 41 43 43 42  1 39 52 63  1 44 59 56 58 46 43 56  6  1 46 43\n",
      " 39 56  1 51 43  1 57 54 43 39 49  8  0  0 13 50 50 10  0 31 54 43 39 49\n",
      "  6  1 57 54 43 39 49  8  0  0 18 47 56 57 58  1 15 47 58 47 64 43 52 10\n",
      "  0 37 53 59]\n",
      "[47 56 57 58  1 15 47 58 47 64 43 52 10  0 14 43 44 53 56 43  1 61 43  1\n",
      " 54 56 53 41 43 43 42  1 39 52 63  1 44 59 56 58 46 43 56  6  1 46 43 39\n",
      " 56  1 51 43  1 57 54 43 39 49  8  0  0 13 50 50 10  0 31 54 43 39 49  6\n",
      "  1 57 54 43 39 49  8  0  0 18 47 56 57 58  1 15 47 58 47 64 43 52 10  0\n",
      " 37 53 59  1]\n",
      "[39 56 43  1 39 50 50  1 56 43 57 53 50 60 43 42  1 56 39 58 46 43 56  1\n",
      " 58 53  1 42 47 43  1 58 46 39 52  1 58 53  1 44 39 51 47 57 46 12  0  0\n",
      " 13 50 50 10  0 30 43 57 53 50 60 43 42  8  1 56 43 57 53 50 60 43 42  8\n",
      "  0  0 18 47 56 57 58  1 15 47 58 47 64 43 52 10  0 18 47 56 57 58  6  1\n",
      " 63 53 59  1]\n",
      "[56 43  1 39 50 50  1 56 43 57 53 50 60 43 42  1 56 39 58 46 43 56  1 58\n",
      " 53  1 42 47 43  1 58 46 39 52  1 58 53  1 44 39 51 47 57 46 12  0  0 13\n",
      " 50 50 10  0 30 43 57 53 50 60 43 42  8  1 56 43 57 53 50 60 43 42  8  0\n",
      "  0 18 47 56 57 58  1 15 47 58 47 64 43 52 10  0 18 47 56 57 58  6  1 63\n",
      " 53 59  1 49]\n"
     ]
    }
   ],
   "source": [
    "seq_dataset = seq_dataset.map(split_input_target)\n",
    "\n",
    "for item_input, item_output in seq_dataset.take(2):\n",
    "    print(item_input.numpy())\n",
    "    print(item_output.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 给dataset做变换\n",
    "batch_size = 64\n",
    "buffer_size = 10000\n",
    "\n",
    "seq_dataset = seq_dataset.shuffle(buffer_size).batch(batch_size, drop_remainder=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型构建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (64, None, 256)           16640     \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (64, None, 1024)          5246976   \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (64, None, 65)            66625     \n",
      "=================================================================\n",
      "Total params: 5,330,241\n",
      "Trainable params: 5,330,241\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "vocab_size = len(vocab) # 词表大小\n",
    "embedding_dim = 256\n",
    "rnn_units = 1024\n",
    "\n",
    "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n",
    "    model = keras.models.Sequential([\n",
    "        keras.layers.Embedding(vocab_size, embedding_dim, batch_input_shape = [batch_size, None]),\n",
    "        # 每一步都要返回全部序列，以便和输出的序列做loss计算\n",
    "        # 调参：stateful = True , recurrent_initializer = 'glorot_uniform'\n",
    "        keras.layers.LSTM(units=rnn_units, stateful = True,\n",
    "                          recurrent_initializer = 'glorot_uniform', return_sequences=True),\n",
    "        keras.layers.Dense(vocab_size)\n",
    "    ])\n",
    "    return model\n",
    "\n",
    "model = build_model(\n",
    "    vocab_size=vocab_size,\n",
    "    embedding_dim=embedding_dim,\n",
    "    rnn_units=rnn_units,\n",
    "    batch_size=batch_size)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 采样测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 100, 65)\n"
     ]
    }
   ],
   "source": [
    "# 没经过训练，已经可以进行预测，看一下model的输出\n",
    "for input_example_batch, target_example_batch in seq_dataset.take(1):\n",
    "    example_batch_predictions = model(input_example_batch)\n",
    "    print(example_batch_predictions.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[48]\n",
      " [ 2]\n",
      " [31]\n",
      " [ 5]\n",
      " [33]\n",
      " [54]\n",
      " [ 1]\n",
      " [48]\n",
      " [ 1]\n",
      " [36]\n",
      " [26]\n",
      " [ 6]\n",
      " [13]\n",
      " [48]\n",
      " [37]\n",
      " [41]\n",
      " [59]\n",
      " [38]\n",
      " [46]\n",
      " [60]\n",
      " [40]\n",
      " [27]\n",
      " [38]\n",
      " [54]\n",
      " [50]\n",
      " [39]\n",
      " [41]\n",
      " [61]\n",
      " [53]\n",
      " [ 7]\n",
      " [ 3]\n",
      " [45]\n",
      " [57]\n",
      " [16]\n",
      " [62]\n",
      " [45]\n",
      " [17]\n",
      " [ 5]\n",
      " [59]\n",
      " [14]\n",
      " [44]\n",
      " [58]\n",
      " [58]\n",
      " [62]\n",
      " [ 5]\n",
      " [ 4]\n",
      " [30]\n",
      " [16]\n",
      " [ 5]\n",
      " [11]\n",
      " [54]\n",
      " [42]\n",
      " [62]\n",
      " [47]\n",
      " [48]\n",
      " [53]\n",
      " [57]\n",
      " [40]\n",
      " [28]\n",
      " [28]\n",
      " [53]\n",
      " [62]\n",
      " [26]\n",
      " [47]\n",
      " [52]\n",
      " [57]\n",
      " [47]\n",
      " [ 6]\n",
      " [ 9]\n",
      " [ 6]\n",
      " [ 9]\n",
      " [57]\n",
      " [23]\n",
      " [62]\n",
      " [18]\n",
      " [10]\n",
      " [ 9]\n",
      " [48]\n",
      " [27]\n",
      " [59]\n",
      " [23]\n",
      " [ 2]\n",
      " [61]\n",
      " [ 6]\n",
      " [62]\n",
      " [ 2]\n",
      " [14]\n",
      " [23]\n",
      " [ 1]\n",
      " [16]\n",
      " [39]\n",
      " [ 3]\n",
      " [50]\n",
      " [32]\n",
      " [39]\n",
      " [18]\n",
      " [49]\n",
      " [14]\n",
      " [ 1]\n",
      " [ 7]], shape=(100, 1), dtype=int64)\n",
      "tf.Tensor(\n",
      "[48  2 31  5 33 54  1 48  1 36 26  6 13 48 37 41 59 38 46 60 40 27 38 54\n",
      " 50 39 41 61 53  7  3 45 57 16 62 45 17  5 59 14 44 58 58 62  5  4 30 16\n",
      "  5 11 54 42 62 47 48 53 57 40 28 28 53 62 26 47 52 57 47  6  9  6  9 57\n",
      " 23 62 18 10  9 48 27 59 23  2 61  6 62  2 14 23  1 16 39  3 50 32 39 18\n",
      " 49 14  1  7], shape=(100,), dtype=int64)\n"
     ]
    }
   ],
   "source": [
    "# 随机采样 random sampling 或者贪心采样 greedy 获得字符，从而生成一段话\n",
    "sample_indices = tf.random.categorical(logits=example_batch_predictions[0], num_samples=1)\n",
    "print(sample_indices) # 采样后的维度 (100, 65) -> (100, 1)\n",
    "\n",
    "# 将维度展平\n",
    "sample_indices = tf.squeeze(sample_indices, axis = -1)\n",
    "print(sample_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: \"ust not simple Henry nor his oaths.\\n\\nMONTAGUE:\\nBrother, I go; I'll win them, fear it not:\\nAnd thus m\"\n",
      "\n",
      "Output: \"st not simple Henry nor his oaths.\\n\\nMONTAGUE:\\nBrother, I go; I'll win them, fear it not:\\nAnd thus mo\"\n",
      "\n",
      "Predictions: \"j!S'Up j XN,AjYcuZhvbOZplacwo-$gsDxgE'uBfttx'&RD';pdxijosbPPoxNinsi,3,3sKxF:3jOuK!w,x!BK Da$lTaFkB -\"\n"
     ]
    }
   ],
   "source": [
    "# 打印输入、输出、预测\n",
    "print('Input:', repr(''.join(idx2char[input_example_batch[0]])))\n",
    "print()\n",
    "print('Output:', repr(''.join(idx2char[target_example_batch[0]])))\n",
    "print()\n",
    "print('Predictions:', repr(''.join(idx2char[sample_indices])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 100)\n",
      "4.1749406\n"
     ]
    }
   ],
   "source": [
    "# 自定义损失函数，loss从logits里来\n",
    "def loss(labels, logits):\n",
    "    return keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n",
    "\n",
    "model.compile(optimizer = 'adam', loss = loss)\n",
    "\n",
    "example_loss = loss(target_example_batch, example_batch_predictions)\n",
    "print(example_loss.shape)\n",
    "print(example_loss.numpy().mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "    172/Unknown - 56s 324ms/step - loss: 2.5390WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 57s 331ms/step - loss: 2.5390\n",
      "Epoch 2/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.8547WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 55s 320ms/step - loss: 1.8547\n",
      "Epoch 3/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.6151WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 55s 319ms/step - loss: 1.6151\n",
      "Epoch 4/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.4879WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 55s 321ms/step - loss: 1.4879\n",
      "Epoch 5/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.4101WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 56s 324ms/step - loss: 1.4101\n",
      "Epoch 6/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.3551WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 60s 351ms/step - loss: 1.3551\n",
      "Epoch 7/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.3107WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 61s 352ms/step - loss: 1.3107\n",
      "Epoch 8/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.2714WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 61s 353ms/step - loss: 1.2714\n",
      "Epoch 9/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.2349WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 65s 378ms/step - loss: 1.2349\n",
      "Epoch 10/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.1987WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 66s 386ms/step - loss: 1.1987\n",
      "Epoch 11/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.1625WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 65s 379ms/step - loss: 1.1625\n",
      "Epoch 12/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.1245WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 63s 367ms/step - loss: 1.1245\n",
      "Epoch 13/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.0855WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 388ms/step - loss: 1.0855\n",
      "Epoch 14/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.0465WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 61s 355ms/step - loss: 1.0465\n",
      "Epoch 15/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 1.0059WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 65s 381ms/step - loss: 1.0059\n",
      "Epoch 16/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.9640WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 388ms/step - loss: 0.9640\n",
      "Epoch 17/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.9220WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 62s 361ms/step - loss: 0.9220\n",
      "Epoch 18/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.8819WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 387ms/step - loss: 0.8819\n",
      "Epoch 19/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.8426WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 63s 363ms/step - loss: 0.8426\n",
      "Epoch 20/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.8044WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 388ms/step - loss: 0.8044\n",
      "Epoch 21/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.7693WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 61s 355ms/step - loss: 0.7693\n",
      "Epoch 22/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.7356WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.7356\n",
      "Epoch 23/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.7066WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 61s 352ms/step - loss: 0.7066\n",
      "Epoch 24/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.6784WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.6784\n",
      "Epoch 25/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.6529WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 64s 372ms/step - loss: 0.6529\n",
      "Epoch 26/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.6297WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 62s 361ms/step - loss: 0.6297\n",
      "Epoch 27/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.6086WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 392ms/step - loss: 0.6086\n",
      "Epoch 28/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.5900WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 62s 359ms/step - loss: 0.5900\n",
      "Epoch 29/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.5726WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 391ms/step - loss: 0.5726\n",
      "Epoch 30/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.5571WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 388ms/step - loss: 0.5572\n",
      "Epoch 31/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.5448WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 388ms/step - loss: 0.5448\n",
      "Epoch 32/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.5313WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 62s 358ms/step - loss: 0.5313\n",
      "Epoch 33/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.5196WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 389ms/step - loss: 0.5196\n",
      "Epoch 34/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.5110WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.5110\n",
      "Epoch 35/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.5024WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 62s 360ms/step - loss: 0.5024\n",
      "Epoch 36/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4930WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 389ms/step - loss: 0.4930\n",
      "Epoch 37/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4855WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 389ms/step - loss: 0.4855\n",
      "Epoch 38/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4785WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 65s 378ms/step - loss: 0.4785\n",
      "Epoch 39/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4708WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 66s 385ms/step - loss: 0.4708\n",
      "Epoch 40/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4677WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.4677\n",
      "Epoch 41/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4623WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.4623\n",
      "Epoch 42/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4562WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.4562\n",
      "Epoch 43/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4526WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 395ms/step - loss: 0.4526\n",
      "Epoch 44/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4489WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.4489\n",
      "Epoch 45/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4439WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 387ms/step - loss: 0.4439\n",
      "Epoch 46/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4418WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.4418\n",
      "Epoch 47/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4379WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.4379\n",
      "Epoch 48/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4356WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 389ms/step - loss: 0.4356\n",
      "Epoch 49/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4315WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 393ms/step - loss: 0.4315\n",
      "Epoch 50/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4285WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 70s 406ms/step - loss: 0.4285\n",
      "Epoch 51/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4270WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 396ms/step - loss: 0.4270\n",
      "Epoch 52/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4245WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 398ms/step - loss: 0.4245\n",
      "Epoch 53/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4233WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 394ms/step - loss: 0.4233\n",
      "Epoch 54/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4225WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.4225\n",
      "Epoch 55/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4183WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 395ms/step - loss: 0.4183\n",
      "Epoch 56/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4175WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 66s 385ms/step - loss: 0.4175\n",
      "Epoch 57/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4159WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 395ms/step - loss: 0.4159\n",
      "Epoch 58/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4139WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 393ms/step - loss: 0.4139\n",
      "Epoch 59/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4120WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 397ms/step - loss: 0.4120\n",
      "Epoch 60/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4113WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 391ms/step - loss: 0.4113\n",
      "Epoch 61/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4084WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.4084\n",
      "Epoch 62/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4078WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 393ms/step - loss: 0.4078\n",
      "Epoch 63/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4063WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 393ms/step - loss: 0.4064\n",
      "Epoch 64/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4072WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 391ms/step - loss: 0.4072\n",
      "Epoch 65/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4069WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 389ms/step - loss: 0.4069\n",
      "Epoch 66/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4040WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 395ms/step - loss: 0.4040\n",
      "Epoch 67/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4039WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 391ms/step - loss: 0.4039\n",
      "Epoch 68/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4021WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 66s 384ms/step - loss: 0.4021\n",
      "Epoch 69/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4021WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 395ms/step - loss: 0.4021\n",
      "Epoch 70/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4031WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 394ms/step - loss: 0.4031\n",
      "Epoch 71/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4012WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.4012\n",
      "Epoch 72/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3976WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 394ms/step - loss: 0.3976\n",
      "Epoch 73/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3983WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.3983\n",
      "Epoch 74/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.4014WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.4014\n",
      "Epoch 75/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3993WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 389ms/step - loss: 0.3994\n",
      "Epoch 76/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3972WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 391ms/step - loss: 0.3972\n",
      "Epoch 77/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3944WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.3944\n",
      "Epoch 78/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3941WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 391ms/step - loss: 0.3941\n",
      "Epoch 79/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3945WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 394ms/step - loss: 0.3945\n",
      "Epoch 80/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3956WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 393ms/step - loss: 0.3956\n",
      "Epoch 81/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3965WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 392ms/step - loss: 0.3965\n",
      "Epoch 82/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3954WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 391ms/step - loss: 0.3954\n",
      "Epoch 83/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3944WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 395ms/step - loss: 0.3944\n",
      "Epoch 84/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3940WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 63s 364ms/step - loss: 0.3940\n",
      "Epoch 85/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3925WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 69s 401ms/step - loss: 0.3925\n",
      "Epoch 86/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3921WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 391ms/step - loss: 0.3921\n",
      "Epoch 87/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3925WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.3925\n",
      "Epoch 88/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3931WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 389ms/step - loss: 0.3931\n",
      "Epoch 89/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3919WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 395ms/step - loss: 0.3919\n",
      "Epoch 90/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3927WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.3927\n",
      "Epoch 91/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3919WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 391ms/step - loss: 0.3919\n",
      "Epoch 92/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3913WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 393ms/step - loss: 0.3913\n",
      "Epoch 93/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3934WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.3934\n",
      "Epoch 94/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3931WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 393ms/step - loss: 0.3931\n",
      "Epoch 95/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3933WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.3933\n",
      "Epoch 96/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3905WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 393ms/step - loss: 0.3905\n",
      "Epoch 97/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3875WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 394ms/step - loss: 0.3875\n",
      "Epoch 98/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3885WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 68s 396ms/step - loss: 0.3885\n",
      "Epoch 99/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3879WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 67s 390ms/step - loss: 0.3879\n",
      "Epoch 100/100\n",
      "171/172 [============================>.] - ETA: 0s - loss: 0.3882WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
      "172/172 [==============================] - 72s 417ms/step - loss: 0.3882\n"
     ]
    }
   ],
   "source": [
    "# 定义文件保存模型\n",
    "output_dir = 'text_generation_lstm'\n",
    "if not os.path.exists(output_dir):\n",
    "    os.mkdir(output_dir)\n",
    "checkpoint_prefix = os.path.join(output_dir, 'ckpt_{epoch}')\n",
    "\n",
    "# 定义参数\n",
    "checkpoint_callback = keras.callbacks.ModelCheckpoint(\n",
    "    filepath = checkpoint_prefix, save_weights_only = True)\n",
    "epochs = 100\n",
    "callbacks = [\n",
    "    checkpoint_callback,\n",
    "    keras.callbacks.EarlyStopping(patience=10, min_delta=1e-3)\n",
    "]\n",
    "\n",
    "# 训练\n",
    "history = model.fit(seq_dataset, epochs = epochs, callbacks = callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'text_generation_lstm\\\\ckpt_100'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.train.latest_checkpoint(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 载入模型权重，进行文本生成\n",
    "文本生成的流程：\n",
    "- start ch sequence A\n",
    "- A -> model -> b\n",
    "- A.append(b) -> B\n",
    "- B(Ab) -> model -> c\n",
    "- B.append(c) -> C\n",
    "- C(Abc) -> model -> ...一直循环结束"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (1, None, 256)            16640     \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (1, None, 1024)           5246976   \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (1, None, 65)             66625     \n",
      "=================================================================\n",
      "Total params: 5,330,241\n",
      "Trainable params: 5,330,241\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model2 = build_model(\n",
    "    vocab_size,\n",
    "    embedding_dim,\n",
    "    rnn_units,\n",
    "    batch_size = 1)\n",
    "# 加载权重\n",
    "model2.load_weights(tf.train.latest_checkpoint(output_dir))\n",
    "# 定义输入大小\n",
    "model2.build(tf.TensorShape([1, None]))\n",
    "\n",
    "model2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 默认生成1000个字符的文本\n",
    "def generate_text(model, start_string, num_generate = 1000):\n",
    "    # 字符变id\n",
    "    input_eval = [char2idx[ch] for ch in start_string]\n",
    "    # 扩展一个维度，一维变二维\n",
    "    input_eval = tf.expand_dims(input_eval, 0)\n",
    "    \n",
    "    text_generated = []\n",
    "    model.reset_states()\n",
    "    \n",
    "    # temperature > 1, more random\n",
    "    # temperature < 1, more greedy\n",
    "    temperature = 0.5\n",
    "    \n",
    "    # 生成流程：\n",
    "    # 1. model inference -> predictions\n",
    "    # 2. sample -> ch -> text_generated.\n",
    "    # 3. update input_eval\n",
    "    for _ in range(num_generate):\n",
    "        # 模型预测 predictions : [batch_size, input_eval_len, vocab_size]\n",
    "        predictions = model(input_eval)\n",
    "        # predictions softmax:e^xi\n",
    "        # eg: 4,2 e^4/(e^4+e^2) = 0.88, e^2/(e^4+e^2) = 0.12\n",
    "        # 除以2后eg: 2,1 e^2/(e^2+e) = 0.73, e/(e^2+e) = 0.27\n",
    "        # 乘2后，相反\n",
    "        predictions = predictions / temperature\n",
    "        # 将第0维消掉\n",
    "        predictions = tf.squeeze(predictions, 0)\n",
    "        # 随机采样，predictions_ids: [input_eval_len, 1]\n",
    "        # 取出预测值中的最后一个字符 a b c -> b c d ,所以取出d\n",
    "        predicted_id = tf.random.categorical(predictions, num_samples = 1)[-1, 0].numpy()\n",
    "        text_generated.append(idx2char[predicted_id])\n",
    "        # 将input_eval替换成predictions，并扩维\n",
    "        input_eval = tf.expand_dims([predicted_id], 0)\n",
    "        \n",
    "    return start_string + ''.join(text_generated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All:\n",
      "All is a plague did over the tempter,\n",
      "Even to the guilt or this untalued dam\n",
      "Which he the sweet silence to a forget other hungs.\n",
      "\n",
      "ROMEO:\n",
      "Why, then the mere it didst thou know thist woman:\n",
      "And I be goned at the great Aufidius\n",
      "A shining stark and safe, and that the world slain.\n",
      "\n",
      "KING RICHARD III:\n",
      "A horse! a horse! my kingdom for a horse!\n",
      "\n",
      "RICHMOND:\n",
      "God and your father gave thee too remiss;\n",
      "Whilst Bolingbroke, the priest should be content\n",
      "To put as where it seems to be achieved.\n",
      "The breathliot that o'er this most peril of your daughter\n",
      "itself into the triumph day,\n",
      "Which made the false seems than starve, 'tis but a man's;\n",
      "And here I stand, both that his son a strength calls and dooms.\n",
      "\n",
      "CAMILLO:\n",
      "I pray not, and all this is long of brother,\n",
      "A femiod dignities and mild behavior,\n",
      "Am bold to show myself a forward guest\n",
      "Within your house, with no man hearing how fares our cousin;\n",
      "Had he done so to well and think they grow:\n",
      "How far is it that she shall have me at the Tower?\n",
      "\n",
      "BUCKINGHAM:\n",
      "It is fo\n"
     ]
    }
   ],
   "source": [
    "new_text = generate_text(model2, 'All:')\n",
    "print(new_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
